#coding:utf-8  
import sys  
reload(sys)
import md5
import time  
sys.setdefaultencoding('utf-8') 
from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from vipshop.items import VipshopItem
from scrapy.contrib.loader import XPathItemLoader
from scrapy.http import Request
from scrapy import log

class VipshopSpider(BaseSpider):
    fileNameIndex = 1
    name = "vipshop"
    allowed_domains = ["www.vipshop.com"]
    start_urls = [
        "http://www.vipshop.com/gz_club.html"
    ]
    DOWNLOAD_DELAY = 1
    id_output = 0

    def parse (self, response):
        hxs_bands = HtmlXPathSelector(response)
        bandUrls = hxs_bands.select('/html/body').select('.//div[@class="ind_brand"]').select('.//a[contains(@target,"_blank")]/@href')
        integer = 1
        for bandUrl in bandUrls:
            if integer % 2 == 1:
                yield Request("http://www.vipshop.com"+bandUrl.extract(), callback=self.parse_band)
            integer = integer + 1

    def parse_band (self, response):
        hxs_products = HtmlXPathSelector(response)
        bandProducts = hxs_products.select('/html/body').select('.//dd[@class="ddTit"]').select('.//a[contains(@target,"_blank")]/@href')
        for bandProduct in bandProducts:
            yield Request("http://www.vipshop.com/"+bandProduct.extract(), callback=self.parse_product)

    def parse_product (self, response):
        print response.url
        url = response.url
        content = response.body
        key = md5.new()
        key.update(content)
        resultDict = {}
        resultDict['url'] = url
        resultDict['content'] = content
        resultDict['md5'] = key.hexdigest()
        resultDict['time'] = time.time()
        open("./data/vipshop" + str(self.fileNameIndex) + ".html",'wb').write(str(resultDict))
        self.fileNameIndex = self.fileNameIndex+1